package com.taoqi.service.common;

import com.sleepycat.je.tree.IN;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.helper.HttpConnection.KeyVal;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;


/**
 * 代理ip抓取类
 *
 * @author xxx
 * 2014年12月22日 下午3:12:07
 * @version V1.0
 */
public class ProxyIpFetcher {
    private static Logger LOGGER = LoggerFactory.getLogger(ProxyIpFetcher.class);

    private static ExecutorService threadPool = Executors.newFixedThreadPool(15);


    public static void main(String[] args) {
        //KeyVal kv = ProxyIpFetcher.fetchOne("http://blog.csdn.net/whos2002110/article/details/42773081",new String[]{"爬虫"});
        List<ProxyReq> kv2= ProxyIpFetcher.fetchAll("http://blog.csdn.net/whos2002110/article/details/42773081",new String[]{"爬虫"});
        System.out.println("找到一个可用代理：" + kv2);
    }

    /**
     *
     * @author xxx
     * 2014年12月24日 下午1:41:51
     * @version V1.0
     */
    private static class ValidateCallable implements Callable<KeyVal> {

        String ip;
        String port;
        String targetUrl;
        String[] keyWords;

        public ValidateCallable(String ip, String port, String targetUrl, String[] keyWords) {
            this.ip = ip;
            this.port = port;
            this.targetUrl = targetUrl;
            this.keyWords = keyWords;
        }

        @Override
        public KeyVal call() throws Exception {
            //使用代理请求页面源代码
            String htmlStr = ProxyIpFetcher.get(targetUrl, ip, port);
            Document document = Jsoup.parse(htmlStr);
            Elements kwMetas = document.head().select("meta[name=keywords]");
            if (kwMetas != null && !kwMetas.isEmpty()) {
                Element kwMate = kwMetas.first();
                String kwCont = kwMate.attr("content");

                boolean contains = false;
                for (String kw : keyWords) {
                    LOGGER.info(kw);
                    if (contains = kwCont.contains(kw)) {
                        break ;
                    }
                }
                if (contains) {
                    //包含关键时认为此代理可用
                    LOGGER.info("ip: " + ip + "port: " + port);
                    System.out.println("ip: " + ip + "port: " + port);
                    return KeyVal.create(ip, port);
                }
            }
            return null;
        }

    }

    /**
     * 获取一个可用的代理ip
     * @param targetUrl 代理要访问的网站
     * @param keyWords 目标页面包含的关键字(用以验证代理确实请求到目标页面)
     * @return ip port pair
     */
    public static KeyVal fetchOne(final String targetUrl, final String[] keyWords) {
        String dlWebUrl = "http://www.kuaidaili.com/free/inha/1";
        Document doc = null;
        try {
            doc = Jsoup.connect(dlWebUrl).timeout(5000).get();
        } catch (Exception e) {
            LOGGER.error("connet {} fail", dlWebUrl);
            return null;
        }
        Element listDiv = doc.getElementById("list");
        Elements trs = listDiv.select("table tbody tr");

        List<Future<KeyVal>> futureList = new ArrayList<Future<KeyVal>>(trs.size());
        for (final Element tr : trs) {
            String ip = tr.child(0).text();
            String port = tr.child(1).text();
            futureList.add(threadPool.submit(new ValidateCallable(ip, port,
                    targetUrl, keyWords)));
        }
        for (int i = 0; i < 3; i++) {
            Thread.yield();//尝试启动验证线程
        }

        for (Future<KeyVal> future : futureList) {
            try {
                KeyVal kv = future.get();
                if (kv == null) {
                    //继续验证,直到获取一个可用的代理
                    continue ;
                }
                return kv;
            } catch (InterruptedException e) {
                LOGGER.error(e.getMessage());
                //i don't what happened, so let it go.
            } catch (ExecutionException e) {
                //call执行异常, 记录日志
                LOGGER.error(e.getMessage());
            } finally {
                if (null != threadPool) {
                    if(!threadPool.isShutdown()){
                        threadPool.shutdown();
                    }
                }
            }
        }
        return null;
    }

    public static List<ProxyReq> fetchAll(final String targetUrl, final String[] keyWords) {
        String dlWebUrl = "http://www.kuaidaili.com/free/inha/1";
        Document doc = null;
        try {
            doc = Jsoup.connect(dlWebUrl).timeout(5000).get();
        } catch (Exception e) {
            LOGGER.error("connet {} fail", dlWebUrl);
            return null;
        }
        Element listDiv = doc.getElementById("list");
        Elements trs = listDiv.select("table tbody tr");

        List<Future<KeyVal>> futureList = new ArrayList<Future<KeyVal>>(trs.size());
        for (final Element tr : trs) {
            String ip = tr.child(0).text();
            String port = tr.child(1).text();
            futureList.add(threadPool.submit(new ValidateCallable(ip, port,
                    targetUrl, keyWords)));
        }
        for (int i = 0; i < 3; i++) {
            Thread.yield();//尝试启动验证线程
        }
        List<ProxyReq> reqs = new ArrayList();
        ProxyReq req = null;
        for (Future<KeyVal> future : futureList) {
            try {
                KeyVal kv = future.get();
                if (kv == null) {
                    //继续验证,直到获取一个可用的代理
                    continue ;
                } else {
                    req = new ProxyReq(kv.key(),Integer.valueOf(kv.value()),"","");
                    reqs.add(req);
                }
            } catch (InterruptedException e) {
                LOGGER.error(e.getMessage());
                //i don't what happened, so let it go.
            } catch (ExecutionException e) {
                //call执行异常, 记录日志
                LOGGER.error(e.getMessage());
            } finally {
                if (null != threadPool) {
                    if(!threadPool.isShutdown()){
                        threadPool.shutdown();
                    }
                }
            }
        }
        return reqs;
    }

    public static String get(String urlToRequest,String proxyIP, String port) {
        HttpClientUtils clients = HttpClientUtils.init().keepLive().openDubug().setProxy(proxyIP, Integer.valueOf(port),"","").build();
        HttpGet get = new HttpGet(urlToRequest);
        Long startTs = System.currentTimeMillis();
        List<NameValuePair> nvps = new ArrayList<NameValuePair>();
        try {
            get.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
            get.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
            get.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
            get.setHeader("Accept-Encoding", "gzip, deflate");
            get.setHeader("Connection", "keep-alive");
            get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko");
            get.setHeader("Upgrade-insecure-Requests", "1");
        } catch (Exception e) {
            e.printStackTrace();
        }
        try {
            String responseString = null;
            CloseableHttpResponse response = null;
            try {
                response = HttpClientUtils.httpclient.execute(get);
                if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
                    HttpEntity entity = response.getEntity();
                    if (null != entity) {
                        responseString = EntityUtils.toString(entity, "UTF-8");
                    }
                    return responseString;
                } else {
                    HttpEntity entity = response.getEntity();

                    if (null != entity) {
                        responseString = EntityUtils.toString(entity, "UTF-8");
                    }
                }
            } catch (IOException e) {
            } finally {
            }
            Long endTs = System.currentTimeMillis();
            Long currentMethodCallTime = endTs - startTs;
            if (currentMethodCallTime > 5000) {
                LOGGER.warn("url:{},call time {} ms", urlToRequest, currentMethodCallTime);
                LOGGER.info("所有存活线程=" + Thread.getAllStackTraces().size());
            }
            return responseString;
        } catch (Exception e) {
            LOGGER.error("[HttpClientUtils][post][Unsupported Encoding Exception]", e);
        }
        return null;
    }
}